package com.chance.cc.crawler.development.scripts.autohome.carSeriesDistributor;

import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Html;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.time.DateUtils;

import java.text.ParseException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;

public class AutoHomeCarSeriesDistributorCrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(AutoHomeCarSeriesDistributorCrawlerScript.class);//日志
    private static final String DOMAIN = "autohome";
    private static final String SITE = "carSeriesDistributor";
    private static final String REQUEST_AGAIN_TAG = DOMAIN + "_request_again";

    //https://dealer.autohome.com.cn/DealerList/GetAreasAjax?provinceId=0&cityId=110100&brandid=0&manufactoryid=0&seriesid=0&isSales=0
    private static final String City = "https://dealer.autohome.com.cn/DealerList/GetAreasAjax\\?provinceId=0&cityId=110100&brandid=0&manufactoryid=0&seriesid=0&isSales=0";//动态获取城市链接
    private static final String FirstUrl = "https://dealer.autohome.com.cn/shanghai/";//动态获取品牌网址
    private static final String PagePrefix = "https://dealer.autohome.com.cn";//网址
    //https://dealer.autohome.com.cn/shanghai/0/0/0/0/4/1/0/0.html
    private static final String NextPageRegulars = "https://dealer.autohome.com.cn/\\S*/0/\\d*/0/0/\\d*/\\d*/0/0.html";//下一页链接
    //https://dealer.autohome.com.cn/15882/#pvareaid=2113601  https://dealer.autohome.com.cn/101068/price.html#pvareaid=2113602
    private static final String ShopRegulars = "https://dealer.autohome.com.cn/\\S*/#pvareaid=\\S*";//门店链接
    //https://dealer.autohome.com.cn/15882/newslist.html https://dealer.autohome.com.cn/102770/newslist.html
    private static final String PromotionRegulars = "https://dealer.autohome.com.cn/\\S*/newslist.html";//促销链接

    //https://dealer.autohome.com.cn/2111236/informationList.html
    private static final String InformationRegulars = "https://dealer\\.autohome\\.com\\.cn/\\S*/informationList\\.html";//新闻资讯链接
    //https://dealer.autohome.com.cn/2105083/informationList_c0_s0_p2.html#newlist
    private static final String NextInformationRegulars = "https://dealer\\.autohome\\.com\\.cn/\\S*/informationList_c0_s0_p\\S*\\.html#newlist";//优惠下一页链接

    //https://dealer.autohome.com.cn/250/newslist_c2_s0.html#title
    private static final String PreferentialRegulars = "https://dealer\\.autohome\\.com\\.cn/\\S*/newslist_c2_s0\\.html#title";//优惠链接和下一页链接
    //https://dealer.autohome.com.cn/250/newslist_c2_s0_p2.html#newlist
    private static final String NextPreferentialRegulars = "https://dealer\\.autohome\\.com\\.cn/\\S*/newslist_c2_s0_p\\S*\\.html#newlist";//优惠下一页链接
    //https://dealer.autohome.com.cn/250/newslist_c3_s0.html#title
    private static final String NewCarRegulars = "https://dealer\\.autohome\\.com\\.cn/\\S*/newslist_c3_s0\\S*\\.html#title";//新车链接
    //https://dealer.autohome.com.cn/250/newslist_c2_s0_p2.html#newlist
    private static final String NextNewCarRegulars = "https://dealer\\.autohome\\.com\\.cn/\\S*/newslist_c3_s0_p\\S*\\.html#newlist";//新车下一页链接
    //https://dealer.autohome.com.cn/102770/news_704073708.html?siteID=49&amp;pvareaid=2015141
    //public static final String DetailstPreferentialAndNewCarRegulars = "https://dealer\\.autohome\\.com\\.cn/\\S*/news_\\S*\\.html\\?siteID=49&amp;pvareaid=2015141";//优惠和汽车详情
    //https://dealer.autohome.com.cn/News/GetNewsContent?dealerId=5028&newsid=704248319&t=template
    private static final String ContentPreferentialRegulars = "https://dealer.autohome.com\\.cn/News/GetNewsContent\\?dealerId=\\S*&newsid=\\S*&t=template";//优惠内容详情
    //https://dealer.autohome.com.cn/News/GetNewsContent?dealerId=5028&newsid=704248319
    private static final String ContentNewCarRegulars = "https://dealer.autohome.com.cn/News/GetNewsContent\\?dealerId=\\S*&newsid=\\d*";//汽车内容详情+新闻资讯内容详情

    @Override
    public String domain() {
        return DOMAIN;
    }//脚本domain定义:识别具体的脚本

    @Override
    public void initUrlRegulars() {//进入脚本的正则列表
        addUrlRegular(City);
        addUrlRegular(FirstUrl);
        addUrlRegular(NextPageRegulars); //下一页
        addUrlRegular(ShopRegulars); //门店
        addUrlRegular(PromotionRegulars);//促销
        addUrlRegular(InformationRegulars);//新闻资讯
        addUrlRegular(NextInformationRegulars);//新闻资讯下一页
        addUrlRegular(PreferentialRegulars);//优惠
        addUrlRegular(NextPreferentialRegulars);//优惠下一页
        addUrlRegular(NewCarRegulars);//新车
        addUrlRegular(NextNewCarRegulars);//新车下一页
//        addUrlRegular(DetailstPreferentialAndNewCarRegulars);//详情页
        addUrlRegular(ContentNewCarRegulars);//详情页内容
        addUrlRegular(ContentPreferentialRegulars);//详情页内容
    }

    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        String site = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        return SITE.equals(site);
    }//是否执行脚本 ： 输入数据检查，合格的才进入脚本

    //获取客户需要的信息
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")) {
                this.initKeyWord(crawlerRequestRecord, supportSourceRecord);
            }
        }
        return prepareLinks;
    }

    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();//获取页面
        Json json = internalDownloadPage.getJson();//获取json
        String msg = json.jsonPath($_type + ".msg").get();//判断是否成功
        if (!"success".equals(msg)) {
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        try {
            Map<String, List<String>> cityAndBrandIdMap = new HashMap();//保存用户信息
            List<String> brandIdList = new ArrayList<>();//保存品牌
            List<String> all1 = json.jsonPath($_type + ".content").all();//获取内容
            for (String data : all1) {//遍历key
                JSONObject jsonObject = JSONObject.parseObject(data);
                String keyword = jsonObject.getString("keyword");//获取key值
                String[] trimUrl = Pattern.compile("[^0-9a-zA-z\\u4e00-\\u9fa5]").matcher(keyword).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ");//获取品牌id
                brandIdList.clear();//清空汽车
                String city = trimUrl[0];
                for (int i = 1; i < trimUrl.length; i++) {//保存城市和汽车
                    brandIdList.add(trimUrl[i]);
                }
                cityAndBrandIdMap.put(city, brandIdList);
            }
            crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("cityAndBrandId", cityAndBrandIdMap);//用户需求放入record中
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    //解析链接方法
    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList();//创建脚本数组
//        判断页面是否加载成功
        String requestUrl = crawlerRequestRecord.getHttpRequest().getUrl();
        if (httpPage.getStatusCode() != 200 || (!httpPage.isDownloadSuccess())) {
            log.error("download page url == {null} error status is {}", requestUrl, httpPage.getStatusCode());
            if (httpPage.getStatusCode() != 404) {//如果没有成功的下载  进行重新下载
                this.requestAgainCrawlerRecord(parsedLinks, crawlerRequestRecord);
                crawlerRequestRecord.setNeedWashPage(false);
                return parsedLinks;
            }
        }

        Map<String, String> carNames = new HashMap();//保存动态获取汽车
        Map<String, String> cityNames = new HashMap();//保存动态获取城市
        if (crawlerRequestRecord.getHttpRequest().getUrl().matches(FirstUrl) || crawlerRequestRecord.getHttpRequest().getUrl().matches(City)) {//解析动态网页
            request(crawlerRequestRecord, httpPage, parsedLinks, carNames, cityNames);
        } else if (crawlerRequestRecord.getHttpRequest().getUrl().matches(NextPageRegulars)) {//解析下一页和门店
            shopRequest(crawlerRequestRecord, httpPage, parsedLinks);
        } else if (crawlerRequestRecord.getHttpRequest().getUrl().matches(ShopRegulars)) {//解析促销和新闻资讯
            promotionRequest(crawlerRequestRecord, httpPage, parsedLinks);
        } else if (crawlerRequestRecord.getHttpRequest().getUrl().matches(PromotionRegulars)) {//解析优惠和新车
            preferentialAndNewCarRequest(crawlerRequestRecord, httpPage, parsedLinks);
        } else if (crawlerRequestRecord.getHttpRequest().getUrl().matches(InformationRegulars) || crawlerRequestRecord.getHttpRequest().getUrl().matches(NextInformationRegulars)) {//新闻资讯详情和下一页
            detailsInformationRequest(crawlerRequestRecord, httpPage, parsedLinks);
        } else if (crawlerRequestRecord.getHttpRequest().getUrl().matches(PreferentialRegulars) || crawlerRequestRecord.getHttpRequest().getUrl().matches(NextPreferentialRegulars)) {//优惠详情和下一页
            detailsPreferentialRequest(crawlerRequestRecord, httpPage, parsedLinks);
        } else if (crawlerRequestRecord.getHttpRequest().getUrl().matches(NewCarRegulars) || crawlerRequestRecord.getHttpRequest().getUrl().matches(NextNewCarRegulars)) {//新车详情和下一页
            detailsNewCarRequest(crawlerRequestRecord, httpPage, parsedLinks);
        }
        return parsedLinks;
    }

    //解析动态地址和品牌
    public void request(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks, Map<String, String> carNames, Map<String, String> cityNames) {//促销方法
        //        解析汽车品牌名
        if (crawlerRequestRecord.getHttpRequest().getUrl().matches(FirstUrl)) {
            List<Selectable> nodes = httpPage.getHtml().xpath("//li[@class=\"row row-hide data-brand-item\"]").nodes();//所有品牌
            for (Selectable node : nodes) {
                List<String> carName = node.xpath(".//a[@class=\"item\"]/text()").all();//品牌名
                List<String> url = node.xpath(".//a[@class=\"item\"]/@href").all();//品牌id
                for (int i = 0; i < carName.size(); i++) {//保存到map数组中
                    String[] trimUrl = Pattern.compile("[^0-9]").matcher(url.get(i)).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ");
                    String brand = trimUrl[1];
                    String name = carName.get(i);
                    carNames.put(name, brand);
                }
            }
            crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("carNames", carNames);//保存品牌信息
            String cityUrl = "https://dealer.autohome.com.cn/DealerList/GetAreasAjax?provinceId=0&cityId=110100&brandid=0&manufactoryid=0&seriesid=0&isSales=0";//动态城市链接
            CrawlerRequestRecord cityRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRequestRecord)
                    .recordKey(cityUrl)
                    .httpUrl(cityUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyResultTags()
                    .copyBizTags()//拷贝上一个自定义标签
                    .build();
            parsedLinks.add(cityRecord);
        }

        //        解析城市名
        if (crawlerRequestRecord.getHttpRequest().getUrl().matches(City)) {
            KVTag carnames = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("carNames");//获取动态汽车名
            Map<String, String> carNamesVal = (Map<String, String>) carnames.getVal();

            List<String> name = httpPage.getJson().jsonPath($_type + "..Name").all();//城市名
            List<String> pinyin = httpPage.getJson().jsonPath($_type + "..Pinyin").all();//城市拼音
            for (int i = 0; i < name.size(); i++) {//保存到map数组中
                cityNames.put(name.get(i), pinyin.get(i));
            }
            KVTag cityAndBrandId = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("cityAndBrandId");//获取用户信息
            Map<String, List<String>> brandIds = (Map<String, List<String>>) cityAndBrandId.getVal();


            for (String city : brandIds.keySet()) {//key用户城市名
                String cityPinyin = cityNames.get(city);//城市名转换拼音
                List<String> brands = brandIds.get(city);//品牌
                for (int i = 0; i < brands.size(); i++) {
                    String brandNumber = carNamesVal.get(brands.get(i));//品牌转换为id数字
                    if (cityPinyin == null || brandNumber == null) {
                        log.error("没有这个城市或品牌" + cityPinyin, brandNumber);
                    } else {
                        String url = "https://dealer.autohome.com.cn/" + cityPinyin + "/0/" + brandNumber + "/0/0/1/1/0/0.html";//得到进入汽车之家主页面
                        CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                                .itemPageRequest(crawlerRequestRecord)
                                .httpUrl(url)
                                .releaseTime(Long.valueOf(System.currentTimeMillis()))
                                .copyBizTags()
                                .build();
                        parsedLinks.add(record);
                    }
                }
            }
        }
    }

    //    主页面解析
    public static void shopRequest(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {//促销方法
        //          下一页解析
        String brandUrl = httpPage.getRequest().getUrl();//获取url
        String[] trimUrl = Pattern.compile("[^0-9]").matcher(brandUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ");//获取车系
        String[] split = brandUrl.split("/");//获取城市
        String cityWord = split[3];//城市
        String nextPageUrl = "https://dealer.autohome.com.cn/" + cityWord + "/0/" + trimUrl[1] + "/0/0/" + (Integer.parseInt(trimUrl[4])+1) + "/1/0/0.html";//网址
        CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                .turnPageRequest(crawlerRequestRecord)
                .recordKey(nextPageUrl)
                .httpUrl(nextPageUrl)
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .build();
        parsedLinks.add(turnPageRequest);

        //          门店解析
        String autoHomeUrl = httpPage.getRequest().getUrl();//获取门店url
        String[] trim = Pattern.compile("[^0-9]").matcher(autoHomeUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ");//截取网址
        String brandId = trim[1];//品牌id
        String seriesId = trim[3];//车系id
        String city = httpPage.getHtml().xpath("//span[@class=\"city-now\"]").get();//城市
        String brand = httpPage.getHtml().xpath("//a[@class=\"item active\"]").get();//品牌
        Map tagBrandDetail = new HashMap();
        tagBrandDetail.put(AutoVMCommonField.Field_Brand_name, brand);//品牌名
        tagBrandDetail.put(AutoVMCommonField.Field_Brand_id, brandId);//品牌id
        tagBrandDetail.put(AutoVMCommonField.Field_Brand_url, brandUrl);//品牌url

        List<Selectable> nodes = httpPage.getHtml().xpath("//li[@class=\"list-item\"]").nodes();//所有门店
        if (nodes.size() > 0) {
            for (Selectable node : nodes) {//获取单个门店
                String shopId = node.xpath("@id").get();//门店id
                String shopStr = node.xpath(".//a[./text()=\"进入店铺\"]/@href").get();
                String shopUrl = null;//店铺url
                if (shopStr != null) {
                    shopUrl = "https:" + shopStr;
                }
                String shopName = node.xpath(".//li[@class=\"tit-row\"]/a/span/text()").get();//店铺名
                String shopPhone = node.xpath(".//span[@class=\"tel\"]").get();//电话
                String shopAddress = node.xpath(".//span[@class=\"info-addr\"]").get();//地址
                Map tagShopDetail = new HashMap();
                tagShopDetail.put(AutoVMCommonField.Field_Shop_id, shopId);
                tagShopDetail.put(AutoVMCommonField.Field_Shop_Url, shopUrl);
                tagShopDetail.put(AutoVMCommonField.Field_Shop_Name, shopName);
                tagShopDetail.put(AutoVMCommonField.Field_Shop_Phone, shopPhone);
                tagShopDetail.put(AutoVMCommonField.Field_Shop_Address, shopAddress);

                if (shopUrl != null && shopUrl != "") {
                    CrawlerRequestRecord shopRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(shopUrl)
                            .httpUrl(shopUrl)
                            .releaseTime(System.currentTimeMillis())
                            .copyBizTags()
                            .build();
//                shopRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Series_id, seriesId);车系
                    shopRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_City, city);
                    shopRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Brand_detail, tagBrandDetail);
                    shopRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Shop_detail, tagShopDetail);
                    shopRecord.tagsCreator().bizTags().addCustomKV("shopUrl", shopUrl);
                    parsedLinks.add(shopRecord);
                }
            }
        }
    }

    //  促销解析
    public static void promotionRequest(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {
        String promotionStr = httpPage.getHtml().xpath("//a[./text()=\"促销信息\"]/@href").get();
        if (promotionStr != null && promotionStr != "") {
            String promotionUrl = PagePrefix + promotionStr;//促销链接
            CrawlerRequestRecord promotionRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRequestRecord)
                    .recordKey(promotionUrl)
                    .httpUrl(promotionUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .build();
            promotionRecord.tagsCreator().bizTags().addCustomKV("promotionUrl", promotionUrl);
            parsedLinks.add(promotionRecord);
        }

        String informationStr = httpPage.getHtml().xpath("//a[./text()=\"新闻资讯\"]/@href").get();
        if (informationStr != null && informationStr != "") {
            String informationUrl = PagePrefix + informationStr;//新闻资讯链接
            CrawlerRequestRecord informationRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRequestRecord)
                    .recordKey(informationUrl)
                    .httpUrl(informationUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .build();
            informationRecord.tagsCreator().bizTags().addCustomKV("informationUrl", informationUrl);
            parsedLinks.add(informationRecord);
        }
    }

    //  优惠和新车解析
    public static void preferentialAndNewCarRequest(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {
        String preferentialStr = httpPage.getHtml().xpath("//a[./text()=\"优惠促销\"]/@href").get();
        if (preferentialStr != null) {
            String preferentialUrl = PagePrefix + preferentialStr;//优惠链接
            CrawlerRequestRecord preferentialRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRequestRecord)
                    .recordKey(preferentialUrl)
                    .httpUrl(preferentialUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .build();
            preferentialRecord.tagsCreator().bizTags().addCustomKV("preferentialUrl", preferentialUrl);
            parsedLinks.add(preferentialRecord);
        }

        String newCarStr = httpPage.getHtml().xpath("//a[./text()=\"新车信息\"]/@href").get();
        if (newCarStr != null) {
            String newCarUrl = PagePrefix + newCarStr;//新车链接
            CrawlerRequestRecord newCarRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRequestRecord)
                    .recordKey(newCarUrl)
                    .httpUrl(newCarUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .build();
            newCarRecord.tagsCreator().bizTags().addCustomKV("newCarUrl", newCarUrl);
            parsedLinks.add(newCarRecord);
        }
    }


    //  详情新闻资讯解析
    public static void detailsInformationRequest(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {
        String pageUrl = httpPage.getHtml().xpath("//a[@class=\"page-next\"]/@href").get(); //新闻资讯下一页
        if (pageUrl != null) {
            String nextPreferentialPageUrl = PagePrefix + pageUrl;
            CrawlerRequestRecord nextPreferentialPageRequest = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .recordKey(nextPreferentialPageUrl)
                    .httpUrl(nextPreferentialPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .build();
            parsedLinks.add(nextPreferentialPageRequest);
        }

        //  新闻资讯解析
        List<Selectable> nodes = httpPage.getHtml().xpath("//dl[@class=\"promot-dl new-dl\"]").nodes();//所有新闻资讯
        if (nodes.size() > 0) {
            for (Selectable node : nodes) {//获取单个新闻资讯url
                String detailsInformationStr = node.xpath("./dd/p/a/@href").get();
                if (detailsInformationStr != null && detailsInformationStr != "") {
                    String detailsInformationUrl = PagePrefix + detailsInformationStr;//新闻资讯链接
                    String title = node.xpath(".//p[@class=\"name font-yh\"]/a/text()").get();//获取标题
                    String pubTime = node.xpath(".//span[@class=\"date-time\"]/text()").get().substring(5);//获取时间
                    if (StringUtils.isBlank(pubTime)) {
                        continue;
                    }
                    //https://dealer.autohome.com.cn/5028/news_704248319.html?siteID=49&pvareaid=2015141
                    String[] trim = Pattern.compile("[^0-9]").matcher(detailsInformationUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ");//获取网址中前两个数字
                    //https://dealer.autohome.com.cn/News/GetNewsContent?dealerId=2111236&newsid=711003151
                    String contentInformationUrl = "https://dealer.autohome.com.cn/News/GetNewsContent?dealerId=" + trim[0] + "&newsid=" + trim[1];//用数字拼接出网址

                    //保存url
                    String shopUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("shopUrl");
                    String informationUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("informationUrl");
                    ArrayList<Map<String, String>> tagCategroyList = new ArrayList<>();
                    HashMap<String, String> tagCategroyMap = new HashMap<>();
                    tagCategroyMap.put("shopUrl", shopUrl);
                    tagCategroyMap.put("informationUrl", informationUrl);
                    tagCategroyMap.put("detailsInformationUrl", detailsInformationUrl);
                    tagCategroyList.add(tagCategroyMap);

                    try {
                        CrawlerRequestRecord contentPreferentialRecord = CrawlerRequestRecord.builder()
                                .itemPageRequest(crawlerRequestRecord)
                                .recordKey(contentInformationUrl)
                                .httpUrl(contentInformationUrl)
                                .releaseTime(DateUtils.parseDate(pubTime, "yyyy-MM-dd").getTime())
                                .resultLabelTag(article)
                                .needWashed(true)
                                .copyBizTags()//拷贝上一个自定义标签
                                .build();
                        contentPreferentialRecord.tagsCreator().bizTags().addCustomKV("detailsInformationUrl", detailsInformationUrl);//新闻资讯url
                        contentPreferentialRecord.tagsCreator().bizTags().addCustomKV("informationTitle", title);//标题
                        contentPreferentialRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Categroy_List, tagCategroyList);//全部标签
                        parsedLinks.add(contentPreferentialRecord);
                    } catch (ParseException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }

    //  详情优惠解析
    public static void detailsPreferentialRequest(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {
        String pageUrl = httpPage.getHtml().xpath("//a[@class=\"page-next\"]/@href").get(); //优惠下一页
        if (pageUrl != null) {
            String nextPreferentialPageUrl = PagePrefix + pageUrl;
            CrawlerRequestRecord nextPreferentialPageRequest = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .recordKey(nextPreferentialPageUrl)
                    .httpUrl(nextPreferentialPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .build();
            parsedLinks.add(nextPreferentialPageRequest);
        }

        //  优惠解析
        List<Selectable> nodes = httpPage.getHtml().xpath("//dl[@class=\"promot-dl\"]").nodes();//所有优惠
        if (nodes.size() > 0) {
            for (Selectable node : nodes) {//获取单个优惠url
                String detailsPreferentialStr = node.xpath("./dd/p/a/@href").get();
                if (detailsPreferentialStr != null) {
                    String detailsPreferentialUrl = PagePrefix + detailsPreferentialStr;//优惠页面链接
                    String title = node.xpath(".//p[@class=\"name font-yh\"]/a/text()").get();//获取标题
                    String overTime = node.xpath(".//span[@class=\"red font-bold\"]/text()").get();//获取优惠结束时间
                    String pubTime = node.xpath(".//span[@class=\"date-time\"]/text()").get().substring(5);//获取时间
                    if (StringUtils.isBlank(pubTime)) {
                        continue;
                    }
                    //https://dealer.autohome.com.cn/5028/news_704248319.html?siteID=49&pvareaid=2015141
                    String[] trim = Pattern.compile("[^0-9]").matcher(detailsPreferentialUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ");//获取网址中前两个数字
                    //https://dealer.autohome.com.cn/News/GetNewsContent?dealerId=5028&newsid=704248319&t=template
                    String contentPreferentialUrl = "https://dealer.autohome.com.cn/News/GetNewsContent?dealerId=" + trim[0] + "&newsid=" + trim[1] + "&t=template";//用数字拼接出网址

                    //保存url
                    String shopUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("shopUrl");
                    String promotionUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("promotionUrl");
                    String preferentialUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("preferentialUrl");
                    ArrayList<Map<String, String>> tagCategroyList = new ArrayList<>();
                    HashMap<String, String> tagCategroyMap = new HashMap<>();
                    tagCategroyMap.put("shopUrl", shopUrl);
                    tagCategroyMap.put("promotionUrl", promotionUrl);
                    tagCategroyMap.put("preferentialUrl", preferentialUrl);
                    tagCategroyMap.put("detailsPreferentialUrl", detailsPreferentialUrl);
                    tagCategroyList.add(tagCategroyMap);

                    try {
                        CrawlerRequestRecord contentPreferentialRecord = CrawlerRequestRecord.builder()
                                .itemPageRequest(crawlerRequestRecord)
                                .recordKey(contentPreferentialUrl)
                                .httpUrl(contentPreferentialUrl)
                                .releaseTime(DateUtils.parseDate(pubTime, "yyyy-MM-dd").getTime())
                                .resultLabelTag(article)
                                .needWashed(true)
                                .copyBizTags()//拷贝上一个自定义标签
                                .build();
                        contentPreferentialRecord.tagsCreator().bizTags().addCustomKV("detailsPreferentialUrl", detailsPreferentialUrl);//优惠url
                        contentPreferentialRecord.tagsCreator().bizTags().addCustomKV("preferentialTitle", title);//标题
                        contentPreferentialRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Remaining_Time, overTime);//结束时间
                        contentPreferentialRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Categroy_List, tagCategroyList);//全部标签
                        parsedLinks.add(contentPreferentialRecord);
                    } catch (ParseException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }

    //  详情页新车解析
    public static void detailsNewCarRequest(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {
        String pageUrl = httpPage.getHtml().xpath("//a[@class=\"page-next\"]/@href").get();//新车下一页
        if (pageUrl != null) {
            String nextNewCarPageUrl = PagePrefix + pageUrl;
            CrawlerRequestRecord nextNewCarturnPageRequest = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .recordKey(nextNewCarPageUrl)
                    .httpUrl(nextNewCarPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .build();
            parsedLinks.add(nextNewCarturnPageRequest);
        }

        //新车解析
        List<Selectable> nodes = httpPage.getHtml().xpath("//dl[@class=\"promot-dl\"]").nodes();//所有新车
        if (nodes.size() > 0) {
            for (Selectable node : nodes) {//获取单个新车url
                String detailsNewCarlStr = node.xpath("./dd/p/a/@href").get();
                if (detailsNewCarlStr != null && detailsNewCarlStr != "") {
                    String detailsNewCarlUrl = PagePrefix + detailsNewCarlStr;//新车内容链接
                    String title = node.xpath(".//p[@class=\"name font-yh\"]/a/text()").get();//获取标题
                    String overTime = node.xpath(".//span[@class=\"red font-bold\"]/text()").get();//获取优惠结束时间
                    String pubTime = node.xpath(".//span[@class=\"date-time\"]/text()").get().substring(5);//获取时间
                    if (StringUtils.isBlank(pubTime)) {
                        continue;
                    }
                    //https://dealer.autohome.com.cn/2111236/news_704520912.html?siteID=49&pvareaid=2015141
                    String[] trim = Pattern.compile("[^0-9]").matcher(detailsNewCarlUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ");
                    //https://dealer.autohome.com.cn/News/GetNewsContent?dealerId=2111236&newsid=704520912
                    String contentNewCarUrl = "https://dealer.autohome.com.cn/News/GetNewsContent?dealerId=" + trim[0] + "&newsid=" + trim[1];

                    //保存url
                    String shopUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("shopUrl");
                    String promotionUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("promotionUrl");
                    String newCarUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("newCarUrl");
                    ArrayList<Map<String, String>> tagCategroyList = new ArrayList<>();
                    HashMap<String, String> tagCategroyMap = new HashMap<>();
                    tagCategroyMap.put("shopUrl", shopUrl);
                    tagCategroyMap.put("promotionUrl", promotionUrl);
                    tagCategroyMap.put("newCarUrl", newCarUrl);
                    tagCategroyMap.put("detailsNewCarlUrl", detailsNewCarlUrl);
                    tagCategroyList.add(tagCategroyMap);

                    try {
                        CrawlerRequestRecord contentNewCarRecord = CrawlerRequestRecord.builder()
                                .itemPageRequest(crawlerRequestRecord)
                                .recordKey(contentNewCarUrl)
                                .httpUrl(contentNewCarUrl)
                                .releaseTime(DateUtils.parseDate(pubTime, "yyyy-MM-dd").getTime())
                                .resultLabelTag(article)
                                .needWashed(true)
                                .copyBizTags()
                                .build();
                        contentNewCarRecord.tagsCreator().bizTags().addCustomKV("detailsNewCarlUrl", detailsNewCarlUrl);//新车url
                        contentNewCarRecord.tagsCreator().bizTags().addCustomKV("newCarTitle", title);//标题
                        contentNewCarRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Remaining_Time, overTime);//结束时间
                        contentNewCarRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Categroy_List, tagCategroyList);//全部标签
                        parsedLinks.add(contentNewCarRecord);
                    } catch (ParseException e) {
                        e.printStackTrace();
                    }
                }
            }
        }

    }

    //  过滤冲洗页面
    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();//解析
        if (crawlerRecord.getHttpRequest().getUrl().matches(ContentPreferentialRegulars)) {
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(washCustomPreferential(crawlerRecord, page));
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(ContentNewCarRegulars)) {
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(washCustomNewCar(crawlerRecord, page));
            }
        }
        return crawlerDataList;
    }

    //  优惠内容清洗
    public List<CrawlerData> washCustomPreferential(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        log.info("优惠内容清洗");
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        try {
            String contentUrl = httpPage.getRequest().getUrl();
            String node = httpPage.getJson().jsonPath("content").get();
            String json = node.substring(1, node.length() - 1);//获取json
            String content = Html.create(json).xpath("//div[@class=\"stencil-text\"]/p").get();//获取优惠内容
            String conditions = Html.create(json).xpath("//em[@class=\"mark mark-gray-hollow\"]/text()").get();//获取优惠条件
            if (StringUtils.isBlank(conditions)) {
                conditions = "无";
            }
            String title = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("preferentialTitle");//获取标题
            String itemUrl = httpPage.getRequest().getUrl();//获取网址
            String articleKey = Pattern.compile("[^0-9]").matcher(itemUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[1];//主要id
            String detailsPreferentialUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("detailsPreferentialUrl");//优惠文章

            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .dataId(StringUtils.joinWith("-", domain(), article, articleKey))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(detailsPreferentialUrl)
                    .addContentKV(AICCommonField.Field_Title, title)
                    .addContentKV(AICCommonField.Field_Content, content.trim())
                    .addContentKV(AutoVMCommonField.Field_Preferential_Terms, conditions.trim())
                    .build();
            crawlerArticleDataList.add(crawlerData);
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
        return crawlerArticleDataList;
    }

    //  新车+新闻资 讯内容清洗
    public List<CrawlerData> washCustomNewCar(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        log.info("新车+新闻资 讯内容清洗");

        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();

        String contentUrl = httpPage.getRequest().getUrl();
        String node = httpPage.getJson().jsonPath("content").get();
        String json = node.substring(1, node.length() - 1);//获取json
        String time = Html.create(json).xpath("//p[@class=\"cont-time\"]/text").get();//获取优惠时间
        if (time != null) {//新车
            try {
                String contnt = Html.create(json).xpath("//div[@class=\"dealertext\"]/p/text()").get();//获取头
                List<String> conditions = Html.create(json).xpath("//table[@class=\"tablecont\"]//td").all();//获取列表
                String pictureBig = Html.create(json).xpath("//p[@class=\"cont-pic\"]/text()").get();//获取图片
                List<String> pictureSmall = Html.create(json).xpath("//p[@class=\"cont-pic-small\"]/text()").all();//获取图片
                String title = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("newCarTitle");//标题
                StringBuffer condition = new StringBuffer();//将文本拼接
                condition.append(contnt);
                for (String c : conditions) {
                    condition.append(c).append(" ");
                }
                StringBuffer pictures = new StringBuffer();//将图片拼接
                pictures.append(pictureBig);
                for (String text : pictureSmall) {
                    pictures.append(text).append(" ");
                }
                String itemUrl = httpPage.getRequest().getUrl();//获取网址
                String articleKey = Pattern.compile("[^0-9]").matcher(itemUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[1];//网址主要id
                String detailsNewCarlUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("detailsNewCarlUrl");//新车文章

                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRequestRecord, httpPage)
                        .dataId(StringUtils.joinWith("-", domain(), article, articleKey))
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                        .url(detailsNewCarlUrl)
                        .releaseTime(System.currentTimeMillis())
                        .addContentKV("time", time)
                        .addContentKV(AICCommonField.Field_Title, title)
                        .addContentKV(AICCommonField.Field_Content, condition.toString().trim())
                        .addContentKV(AICCommonField.Field_Images, pictures.toString().trim())
                        .build();
                crawlerArticleDataList.add(crawlerData);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            return crawlerArticleDataList;
        } else {//新闻资讯
            try {
                log.info("新车+新闻资 讯内容清洗");
                List<String> conditions = Html.create(json).xpath("//p/text()").all();//获取文本
                List<String> pictureSmall = Html.create(json).xpath("//img/@src").all();//获取图片
                String title = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("informationTitle");//标题
                StringBuffer condition = new StringBuffer();//将文本拼接
                for (String c : conditions) {
                    condition.append(c).append(" ");
                }
                StringBuffer pictures = new StringBuffer();//将图片拼接
                for (String text : pictureSmall) {
                    pictures.append(text).append(" ");
                }
                String itemUrl = httpPage.getRequest().getUrl();//获取网址
                String articleKey = Pattern.compile("[^0-9]").matcher(itemUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[1];//网址主要id
                String detailsInformationUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("detailsInformationUrl");//新闻资讯文章

                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRequestRecord, httpPage)
                        .dataId(StringUtils.joinWith("-", domain(), article, articleKey))
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                        .url(detailsInformationUrl)
                        .releaseTime(System.currentTimeMillis())
                        .addContentKV(AICCommonField.Field_Title, title)
                        .addContentKV(AICCommonField.Field_Content, condition.toString().trim())
                        .addContentKV(AICCommonField.Field_Images, pictures.toString().trim())
                        .build();
                crawlerArticleDataList.add(crawlerData);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            return crawlerArticleDataList;
        }
    }

    //最后执行的方法
    @Override
    public void afterExecute(CrawlerRecordContext crawlerRecordContext) {
    }

    /*
     * 重新下载
     * */
    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(REQUEST_AGAIN_TAG)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(REQUEST_AGAIN_TAG).getVal();
            if (count >= 10) {
                log.error("pcauto download page the number of retries exceeds the limit" +
                        ",request url {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        CrawlerRequestRecord crawlerRequestRecord = null;
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get("turn_page_item_request");
        if (type == null) {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        } else {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }

        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(REQUEST_AGAIN_TAG, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }

}